{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.vision import *\n",
    "import torch\n",
    "torch.cuda.set_device(0)\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=UserWarning, module=\"torch.nn.functional\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = untar_data(URLs.CIFAR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For CIFAR datasets\n",
    "tfms = get_transforms(do_flip=False)\n",
    "data = ImageDataBunch.from_folder(path, train = 'train', valid = 'test', bs = 64, size = 32, ds_tfms = tfms).normalize(cifar_stats)\n",
    "\n",
    "# For Imagenet and its subsets\n",
    "# tfms = get_transforms(do_flip=False)\n",
    "# data = ImageDataBunch.from_folder(path, train = 'train', valid = 'val', bs = 64, size = 224, ds_tfms = tfms).normalize(imagenet_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential\n",
       "======================================================================\n",
       "Layer (type)         Output Shape         Param #    Trainable \n",
       "======================================================================\n",
       "Conv2d               [64, 16, 16]         9,408      True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [64, 16, 16]         128        True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [64, 16, 16]         0          False     \n",
       "______________________________________________________________________\n",
       "MaxPool2d            [64, 8, 8]           0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [64, 8, 8]           36,864     True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [64, 8, 8]           128        True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [64, 8, 8]           0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [64, 8, 8]           36,864     True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [64, 8, 8]           128        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [64, 8, 8]           36,864     True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [64, 8, 8]           128        True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [64, 8, 8]           0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [64, 8, 8]           36,864     True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [64, 8, 8]           128        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [64, 8, 8]           36,864     True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [64, 8, 8]           128        True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [64, 8, 8]           0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [64, 8, 8]           36,864     True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [64, 8, 8]           128        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [128, 4, 4]          73,728     True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [128, 4, 4]          256        True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [128, 4, 4]          0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [128, 4, 4]          147,456    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [128, 4, 4]          256        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [128, 4, 4]          8,192      True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [128, 4, 4]          256        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [128, 4, 4]          147,456    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [128, 4, 4]          256        True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [128, 4, 4]          0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [128, 4, 4]          147,456    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [128, 4, 4]          256        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [128, 4, 4]          147,456    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [128, 4, 4]          256        True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [128, 4, 4]          0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [128, 4, 4]          147,456    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [128, 4, 4]          256        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [128, 4, 4]          147,456    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [128, 4, 4]          256        True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [128, 4, 4]          0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [128, 4, 4]          147,456    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [128, 4, 4]          256        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [256, 2, 2]          294,912    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [256, 2, 2]          512        True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [256, 2, 2]          0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [256, 2, 2]          589,824    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [256, 2, 2]          512        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [256, 2, 2]          32,768     True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [256, 2, 2]          512        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [256, 2, 2]          589,824    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [256, 2, 2]          512        True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [256, 2, 2]          0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [256, 2, 2]          589,824    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [256, 2, 2]          512        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [256, 2, 2]          589,824    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [256, 2, 2]          512        True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [256, 2, 2]          0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [256, 2, 2]          589,824    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [256, 2, 2]          512        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [256, 2, 2]          589,824    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [256, 2, 2]          512        True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [256, 2, 2]          0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [256, 2, 2]          589,824    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [256, 2, 2]          512        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [256, 2, 2]          589,824    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [256, 2, 2]          512        True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [256, 2, 2]          0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [256, 2, 2]          589,824    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [256, 2, 2]          512        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [256, 2, 2]          589,824    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [256, 2, 2]          512        True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [256, 2, 2]          0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [256, 2, 2]          589,824    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [256, 2, 2]          512        True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [512, 1, 1]          1,179,648  True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [512, 1, 1]          1,024      True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [512, 1, 1]          0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [512, 1, 1]          2,359,296  True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [512, 1, 1]          1,024      True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [512, 1, 1]          131,072    True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [512, 1, 1]          1,024      True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [512, 1, 1]          2,359,296  True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [512, 1, 1]          1,024      True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [512, 1, 1]          0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [512, 1, 1]          2,359,296  True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [512, 1, 1]          1,024      True      \n",
       "______________________________________________________________________\n",
       "Conv2d               [512, 1, 1]          2,359,296  True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [512, 1, 1]          1,024      True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [512, 1, 1]          0          False     \n",
       "______________________________________________________________________\n",
       "Conv2d               [512, 1, 1]          2,359,296  True      \n",
       "______________________________________________________________________\n",
       "BatchNorm2d          [512, 1, 1]          1,024      True      \n",
       "______________________________________________________________________\n",
       "AdaptiveAvgPool2d    [512, 1, 1]          0          False     \n",
       "______________________________________________________________________\n",
       "AdaptiveMaxPool2d    [512, 1, 1]          0          False     \n",
       "______________________________________________________________________\n",
       "Flatten              [1024]               0          False     \n",
       "______________________________________________________________________\n",
       "BatchNorm1d          [1024]               2,048      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [1024]               0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512]                524,800    True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [512]                0          False     \n",
       "______________________________________________________________________\n",
       "BatchNorm1d          [512]                1,024      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [512]                0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [10]                 5,130      True      \n",
       "______________________________________________________________________\n",
       "\n",
       "Total params: 21,817,674\n",
       "Total trainable params: 21,817,674\n",
       "Total non-trainable params: 0\n",
       "Optimized with 'torch.optim.adam.Adam', betas=(0.9, 0.99)\n",
       "Using true weight decay as discussed in https://www.fast.ai/2018/07/02/adam-weight-decay/ \n",
       "Loss function : FlattenedLoss\n",
       "======================================================================\n",
       "Callbacks functions applied "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn = cnn_learner(data, models.resnet34, metrics = accuracy, pretrained = True)\n",
    "learn.unfreeze()\n",
    "learn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEKCAYAAADTgGjXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hc1bX38e9Sd5G75Cb3igtuAtuYACYmdBNCIAackEDCDQQSWgi55IVg0rn0ci8lEBIg9FAcmgMYG+Mmd9yNZeMuucmSbMkq6/1jRiCErGaNpuj3eZ55PHPOnnPW9khas/c+Z29zd0RERBoiLtwBiIhI9FISERGRBlMSERGRBlMSERGRBlMSERGRBlMSERGRBgt5EjGzeDNbYmbTj7D/IjNbZWYrzey5UMcjIiKNJ6EJzvELYDXQpuoOMxsA/BqY4O77zCy9CeIREZFGEtKWiJllAGcDTxyhyE+Ah919H4C754QyHhERaVyhboncB9wMpB5h/0AAM5sDxAO/dfd3ajpgp06dvHfv3o0Zo4hIzFu0aNFud09r7OOGLImY2TlAjrsvMrNTajj/AOAUIAOYbWbD3H1/lWNdCVwJ0LNnT7KyskIVtohITDKzzaE4bii7syYAk81sE/A8cKqZPVOlzFbgdXcvcfdsYC2BpPIV7v6Yu2e6e2ZaWqMnUhERaaCQJRF3/7W7Z7h7b2AK8IG7T61S7DVgIoCZdSLQvbUxVDGJiEjjavL7RMxsmplNDr58F9hjZquAD4Ffuvuepo5JREQaxqJtKvjMzEzXmIiISP2Y2SJ3z2zs4+qOdRERaTAlERERaTAlERERaTAlERGRCHfocBl3vbuGZVv21164iSmJiIhEuH0HD/Pwh5+xaseBcIfyNUoiIiIRLr+oFIDUlKaYM7d+lERERCJcflEJAKkpiWGO5OuUREREIlx+caAl0jpZLREREamniu6sNurOEhGR+lJ3loiINFiBBtZFRKSh8otKiTNomRQf7lC+RklERCTC5ReV0Do5ATMLdyhfoyQiIhLh8otKI3I8BJREREQiXn5xaUSOh4CSiIhIxMsvKlESERGRhlF3loiINFhBcWlE3q0OSiIiIhEv0BJREhERkXpy9+CYSDPtzjKzeDNbYmbTayjzXTNzM2v0ReRFRKJZcWk5JWXerFsivwBWH2mnmaUCPwfmN0EsIiJRJZLXEoEQJxEzywDOBp6oodidwF+AolDGIiISjb6cfLEZJhHgPuBmoLy6nWY2Cujh7kfs6hIRac6+aIkkN7MxETM7B8hx90VH2B8H3AvcWIdjXWlmWWaWlZub28iRiohEroKKBamaYUtkAjDZzDYBzwOnmtkzlfanAsOAmcEy44A3qhtcd/fH3D3T3TPT0tJCGLKISGRptt1Z7v5rd89w997AFOADd59aaX+eu3dy997BMvOAye6eFaqYRESizYEvVjVsZt1ZR2Jm08xsclOfV0QkGkXyglQATRKVu88EZgaf33aEMqc0RSwiItGkYmC9laY9ERGR+sovKqFFYjyJ8ZH55zoyoxIRESBwdVakdmWBkoiISETLLyqN2Mt7QUlERCSiHYjgyRdBSUREJKLlF5XSRi0RERFpiEhekAqUREREIlokr68OSiIiIhEtktdXByUREZGIVVbuHDxcppaIiIjUX8WUJxoTERGRejsQnME3UidfBCUREZGIVbGWiLqzRESk3iomX9Qd6yIiUm9fLkil7iwREamn/AhfSwSUREREIla+xkRERKShvujOSlZ3loiI1FN+USkJcUZKYuT+qY7cyEREmrmCosCCVGYW7lCOSElERCRC5ReVRPTlvdAEScTM4s1siZlNr2bfDWa2ysyWm9n7ZtYr1PGIiESL/KLSiB4PgaZpifwCWH2EfUuATHc/FngZ+EsTxCMiEhXyI3x9dQhxEjGzDOBs4Inq9rv7h+5+MPhyHpARynhERKJJflEzTyLAfcDNQHkdyl4BvB3acEREokd+hK+vDiFMImZ2DpDj7ovqUHYqkAncdYT9V5pZlpll5ebmNnKkIiKRqbm3RCYAk81sE/A8cKqZPVO1kJlNAm4FJrt7cXUHcvfH3D3T3TPT0tJCGLKISGRwdwqa85iIu//a3TPcvTcwBfjA3adWLmNmo4BHCSSQnFDFIiISbQ6VlFFW7rTW1VlfZWbTzGxy8OVdQGvgJTNbamZvNHU8IiKRKBomXwRokujcfSYwM/j8tkrbJzXF+UVEok20JBHdsS4iEoG+XEtESUREROrpy5aIxkRERKSeomF9dVASERGJSNGwNC4oiYiIRKSK7qzWyWqJiIhIPSmJiIhIg+UXldI6OYH4uMhdkAqUREREIlJ+UUnEt0JASUREJCJFw+SLoCQiIhKRomHyRVASERGJSIH11SP78l5QEhERiUjqzhIRkQbLLy6ljZKIiIg0RDQsjQtKIiIiEaekrJyiknJd4isiIvVXECVriYCSiIhIxImWaeBBSUREJOIcCM7gq+4sERGpty17DwLQvV2LMEdSOyUREZEIsyGnAIB+6a3CHEntlERERCLMhtwCurdrQcskdWdhZvFmtsTMplezL9nMXjCzDWY238x6hzoeEZFItyGngH7prcMdRp00RUvkF8DqI+y7Atjn7v2Be4E/N0E8IiIRq7zc+Sy3gH5pkd+VBSFOImaWAZwNPHGEIucBTwefvwx808wiewUWEZEQ2p53iKKScvqrJQLAfcDNQPkR9ncHtgC4eymQB3SsWsjMrjSzLDPLys3NDVWsIiJhVzGo3j+tmScRMzsHyHH3RTUVq2abf22D+2PununumWlpaY0Wo4hIpPkiiaglwgRgspltAp4HTjWzZ6qU2Qr0ADCzBKAtsDeEMYmIRLTPcgto3zKRjq2Twx1KnYQsibj7r909w917A1OAD9x9apVibwCXBZ9/N1jmay0REZHmYkNOQdS0QiAM94mY2TQzmxx8+Vego5ltAG4AbmnqeEREIkm0JZEmuZPF3WcCM4PPb6u0vQi4sCliEBGJdHsKitl3sIR+UTKoDrpjXUQkYkTboDooiYiIRIwNuUoiIiLSQBtyCmiRGE+3tpE/e28FJRERkQgRmDOrFXFx0TNxh5KIiEiE+CynIKoG1UFJREQkIhQWl7I9ryhqpjupUKckYmb9zCw5+PwUM/u5mbULbWgiIs3HZ1E4qA51b4m8ApSZWX8CNwj2AZ4LWVQiIs1MrCeR8uAsu+cD97n79UDX0IUlItK8bMgpID7O6NUxOtYRqVDXJFJiZhcTmOeqYoXCxNCEJCLS/GzIKaBXx5YkJUTXUHVdo/0RMB74vbtnm1kfoOqMvCIi0kAbcgqiblAd6jh3lruvAn4OYGbtgVR3/1MoAxMRaS5KysrZvOcgpw/tEu5Q6q2uV2fNNLM2ZtYBWAY8ZWb3hDY0EZHmYfOeQkrLPeoG1aHu3Vlt3f0A8B3gKXcfA0wKXVgiIs3Hmp35QPRdmQV1TyIJZtYVuIgvB9ZFRKQRLMzeS4vEeI7p2ibcodRbXZPINOBd4DN3X2hmfYH1oQtLRKT5mJ+9l8ze7UmMj64rs6DuA+svAS9Ver0RuCBUQYmINBd7Cw+zZmc+5xwbnbfe1XVgPcPM/mVmOWa2y8xeMbOMUAcnIhLrFmTvBWBc345hjqRh6tp2egp4A+gGdAfeDG4TEZGjMG/jHlIS4zg2IzqnI6xrEklz96fcvTT4+BuQFsK4RESahfnZexnTq33U3aleoa5R7zazqWYWH3xMBfbU9AYzSzGzBWa2zMxWmtkd1ZTpaWYfmtkSM1tuZmc1pBIiItFo/8HDrNl5gHF9orMrC+qeRC4ncHnvTmAH8F0CU6HUpBg41d1HACOBM8xsXJUyvwFedPdRwBTgkboGLiIS7RZk78UdxkbpeAjUMYm4++fuPtnd09w93d2/TeDGw5re4+5eEHyZGHx41WJAxYXRbYHtdQ9dRCS6zdu4l+SEOEb0aBvuUBrsaDrhbqitQLDraymQA8xw9/lVivwWmGpmW4G3gGuPIh4RkagyP3sPo3u2JzkhPtyhNNjRJJFaV5J39zJ3HwlkAMeb2bAqRS4G/ubuGcBZwD/M7GsxmdmVZpZlZlm5ublHEbKISGTIO1jCqh0HovbS3gpHk0Sqdk0duaD7fmAmcEaVXVcALwbLzAVSgE7VvP8xd89098y0tPBfFHagqIRFm/fywsLPWRuc80ZEpD4WbqoYD+kQ7lCOSo13rJtZPtUnCwNa1PLeNKDE3febWQsCEzb+uUqxz4FvAn8zs2MIJJGIbGrsLTzMHW+uZGH2XrbnFX2xPTHeuPn0wVxxYh/i4mptnImIAIH7Q5IS4hjZIzrvD6lQYxJx99SjOHZX4GkziyfQ4nnR3aeb2TQgy93fAG4EHjez6wkkqx+6e51bOPXx+Z6DzMvew3dGdSehnvPTrNuVzxVPL2TXgWLOHNaFQV1SGdwlle7tWnL3e2v5/VurmbU+l7svHEF6m5RQhC8iMWZ+9l5G9WhHSmL0jocAWIj+ZodMZmamZ2Vl1ft998xYxwPvr6d3x5b8YtIAJo/oTnwdWg7/WbWLXzy/hJbJCTz2/TGM6tn+K/vdnX8u2MK06StpmZTA9acNZOKgNDLat6x3jCLSPBwoKmHkHe9x7akDuP60gU1yTjNb5O6ZjX3cOk3AGAuunzSAYd3acM+MdVz/wjIe+mADP/lGX1omJ1B0uIyDh0spLi0nPs5ITownOT6OTXsK+d+PPmNYt7Y89oMxdG379R48M+OSsT05vk97rnthKf/vtU8B6N2xJRP6d+KkgWmc2L8TrZIb/786J7+I/5u5kRP6dWTSkM6NfnwRCY2sTXspj4HxEGhGLZEK5eXOOyt3cu+MdazPKai1/DnHduWu746gRVLtTU53Z0NOAbPX72bOht3M27iHwsNlJMXHMa5fRyYdk863hnShS9uj6/IqLSvnmXmbufu9deQXlwJwxYl9+NUZg+s9dcLh0nJWbMtjZ14RO/IOsetAEYWHyxjUOZWh3dpwTNc2tSbAnANFtEpOCEmiFIlFd05fxTPzNrPs9m81WXdWqFoizS6JVCgrd9btyicx3khJjKdlUgLJCXGUljuHS8s5XFYOQLe2KZg1bMD8cGk5WZv38sHqHN5fk0P27kJaJsXz5A+Pa9BlfSVl5SzevI9p01excvsBTuzfid+ccwz/nP85T8/dzMge7XjoklF17krbmFvAz55bwuodB77YlpwQR3JCHAeKAsnJDPp0asXonu3J7NWezN7t6dOpNUu37Oc/q3cxY9UuNgSTcXpqMn06tfqi/MmD0uisMSKRrzntno/o0jaFf1wxtsnOqSQS1FhJJBzW78rn6mcXs2XfQZ74wXGcOOBrVzN/RX5RCU9/solVOw6wIaeA7N2FlJQ5ndsk8//OGcLZw7t+keDeWrGDX728nLg449azjuG8Ud1qvIHp9aXb+O9XV5CUEMdvzh7C0O5t6NImhbYtEgHYeaCIldsOsHL7AVZs28+izfvYd7AEgIQ4o7TcSYgzxvbtwCkD0zlcVk727kI27S7ks9yCL8oe07UNpwxKY9Ix6Yzq0V5XsEmztyPvEOP/+AG3nnUMPzmpb5OdV0kkKJqTCMDugmKmPjGfjbsLeXTqGCYOTq+23JqdB7j6mcVk7ymkV4eW9E9vTf/0VAakt+ZbQzuTmpL4tfds3lPIz59fyrIt+0lPTeayE3pz6dietGuZBAS62/IOlfDnd9byzwWfk9mrPQ9cPIpu7Wq8WvuL92bvLiRr8z7W7szn2Iy2nDIwnbYtvx6Hu7N6Rz4z1+Uwc20uizbvo6zc6dQ6mdOGBLr0TujfMarv0hVpqBcXbuHmV5bz7nUnMajL0VwAWz9KIkHRnkQA9hUe5vtPzmftznzu+94ozhzW5Svf0F9etJXfvLaC1JREHrp4VL0mZ3N3Zq/fzeOzNzJ7/W5aJMYzsEsqewqKyc0vprg00E131Sn9uOG0gU2yHGfeoRJmrs3hvVW7mLkmh8LDZbRvmciU43ty6dieupJNmpVrnlvMwk17mffrbza4q7whlESCYiGJQOAP62VPLmDplv20Tk5gaLc2DO/elr2Fh3l1yTbG9+3I/RePJD214WMKq3cc4Kk52WzfX0RaajJpqcl0ap0UGN/oHZ6rQopLy5izYTcvLNzCjFW7AJh0TGemjuvFCf061vseHpFoUlbujPndDCYd05n/uXBEk55bSSQoVpIIwMHDpfx7+Q5WbMtj+dY8Vu84QHFpOT+b2I/rJw2M+T+o2/Yf4tl5m3l+4Rb2Fh6mY6skzhrelXNHdCOzl8ZPJPYs3bKfbz88h/unjOS8kd2b9NxKIkGxlESqKikrp7C49IsxjOaiqKSMmWtzeXP5dt5fvYuiknK6t2vBD8b3YsrxPb8Y7BeJdg++v557/rOORb85jQ6tmvb3XDcbNgOJ8XHNLoEApCTGc8awLpwxrAuFxaX8Z/Uu/rngc/749hruf3893x2TwWUn9KZfWutwhypyVGatz2VYt7ZNnkBCSUlEIkqr5ATOG9md80Z2Z+X2PJ78eBPPL9jC3+dupnu7Fozt24FxfToyvl9HenTQgLxEj/yiEhZ/vp+fntx0l/U2BSURiVhDu7Xl7otG8KszB/H2ip3Mz97DR2tzeXXxNgDOGNqFG741kIGdm+4ySZGG+uSzPZSVO98YEP7lLBqTkohEvPTUFC47oTeXndD7i6ll3ly+gyc/zubdVTs5b0Q3rps0kN6dWoU7VJEjmr0+l1ZJ8YyuMolrtIvty38k5pgZAzqncsNpA5l980T+66R+vLNyJ9+85yPunL6KguBcYiKRZta63Yzv17He89tFutiqjTQr7VslccuZg5l180QuyuzBk3OymXT3R7y1YgfRdtWhxLbNewr5fO9BThoYW11ZoCQiMSA9NYU/fmc4r1x1Ah1aJXH1s4v54VMLeXvFDrbsPaiEImE3a11gwdZYGw8BjYlIDBndsz1vXDOBv8/dzL0z1vFR8Be3XctEhndvyznHduU7ozOaZKoXkco++WwP3du1oHfH2LuiUElEYkpCfByXn9iHS8b2ZO3OfFZsy+PTbXlkbd7Hr15ZwYMfbOCaif35zuiMmOublsjk7szP3svEQelNOldWU1ESkZiUkhjPiB7tGNGjHRD4RZ65Npf73l/PLa8Gksl/ndyX74zOoLUW05IQWp9TwN7CwzGximF19FVMmgUzY+LgdF67+gSe+tFxpLdJ5rbXVzL29//httc/Zf2u/HCHKDFq3sY9AIxvwEJ00SBkX8HMLAWYBSQHz/Oyu99eTbmLgN8CDixz90tCFZOImTFxUDoTB6WzdMt+/j53E88vDNwRP7Bza3p1bEWvDi3p2bElPdq3pHObFLq2TaFdy8SY7IqQ0Ju/cS/d2qaQ0b72dXuiUSjb8cXAqe5eYGaJwMdm9ra7z6soYGYDgF8DE9x9n5lVv0KTSAiM7NGOkT1GcutZx/Bi1lYWbd7L5j2FzF6fS1FJ+VfKJifEMbhLKreePYTj+8Rmt4Q0vsB4yB5OGpAWs19CQpZEPHBdZUHwZWLwUfVay58AD7v7vuB7ckIVj8iRdGydzFWn9AP6AYFf/Nz8YrbsO8SuA0XszCtiR94h3lqxk4sencvFx/fkljMHa3ZhqdVnuQXsLojd8RAI8cC6mcUDi4D+BJLF/CpFBgbLzQHigd+6+zuhjEmkNmZGepsU0tt8dUGw608byL0z1vHXj7P5z+pd3DF5KGcN7xqmKCUazN24F4BxMToeAiEeWHf3MncfCWQAx5vZsCpFEoABwCnAxcATZtau6nHM7EozyzKzrNzc3FCGLHJELZMSuPXsIbxxzYl0bpPM1c8u5onZG8MdlkSw+Rv30KVNCj1jeMbpJrk6y933AzOBM6rs2gq87u4l7p4NrCWQVKq+/zF3z3T3zLS02LvjU6LLsO5tee3qCZw9vCu/+/dqnvw4O9whSQRyd+Zt3Mu4vh1idjwEQphEzCytolVhZi2AScCaKsVeAyYGy3Qi0L2lr3YS8RLi47hvykjOGNqFadNX8fe5m8IdkkSYjbsL2V1QzNgY7sqC0LZEugIfmtlyYCEww92nm9k0M5scLPMusMfMVgEfAr909z0hjEmk0STGx/HAxaM4bUhnbnt9Jc/M2xzukCSCVNwfEsvjIRDaq7OWA6Oq2X5bpecO3BB8iESdpIQ4Hr5kNFc9s4jfvPYpa3fm86szB+sueGHexr2kpybH5HxZlemOdZGjlJQQxyNTR3P5hD48M38zp98764vJH6V5cnfmb9zDuL4dY3o8BJRERBpFckI8t507hJd/egItkuK57MkF3PjiMvIOlYQ7NAmD7N2F5OQXx/T9IRWUREQa0Zhe7Zl+7YlcM7E/ry3dxhn3zWLOht3hDkua2Pzs2L8/pIKSiEgjS0mM56bTB/HqVYFWyaVPzOfO6asoKikLd2jSBMrKnX8t2UZaajJ9O7UKdzghpyQiEiIjerTj39d+gx+M78VfP87m3Ac/ZmNuQe1vlKh274x1LMjey42nDYz58RBQEhEJqRZJ8Uw7bxh/+9Fx7C4o5sd/zyK/SOMksWrGql089OEGvpfZgynH9wx3OE1CSUSkCZwyKJ1HLh3D5j0H+eVLy7XuewzatLuQG15cyvDubbnjvKHhDqfJKImINJHx/TpyyxmDeWflTh6dpYkZYsmhw2X89JlFxMcZj1w6mpTE+HCH1GSURESa0I+/0Yezh3flL++s4RNdtRUzpk1fxdpd+dw/ZRQ9YniyxeooiYg0ITPjz989lr5prbnmn0vYtv9QuEOSo5R3sIRXFm3lkuN7cvLA5jdBrJKISBNrnZzAo98fw+HSci78309YsTUv3CHJUXhz+XYOl5Uz5bjmMZBelZKISBj0S2vN81eOw8z47v99wmtLtoU7JGmgVxdvZWDn1gzr3ibcoYSFkohImAzr3pbXr5nAiB7tuO6Fpfz+36soLSuv/Y0SMTbmFrD48/1cMDqjWdwTUh0lEZEw6tQ6mWd/PJYfjO/F47OzufjxeWzIyQ93WFJHry7eRpzBt0d1D3coYaMkIhJmifFxTDtvGPdcNIJ1uwo48/7Z3DNjnaZJiXDlwelNThyQRuc2KeEOJ2yUREQixHdGZ/D+jSdz9vCuPPD+es66fzbzN2qNtkg1L3sP2/Yf4oLRzbcVAkoiIhGlU+tk7psyiqcvP56S8nKmPD6PP761muJStUoizSuLtpGanMDpQ7uEO5SwUhIRiUAnD0zj3etO4uLje/LorI2c99Ac1uw8EO6wJOjg4VLe/nQHZw3v2qzuTq+OkohIhGqZlMAfzh/OXy/LZHdBMZMfnMMTszdq3q0I8M6nOzl4uIwLxmSEO5SwUxIRiXDfPKYz71x3EicNTON3/17Nj5/OYl/h4XCH1ay9lLWVnh1aclzv9uEOJexClkTMLMXMFpjZMjNbaWZ31FD2u2bmZpYZqnhEolmn1sk8/oMx3DF5KLPX7+asB2aTtWlvuMNqlj5al8vcjXuYOq5ns703pLJQtkSKgVPdfQQwEjjDzMZVLWRmqcDPgfkhjEUk6pkZl53Qm1euOoGkhDi+99g8Hv5wA+Xl6t5qKodLy7njjZX06dSKy07oHe5wIkLIkogHVCzjlhh8VPfTfifwF6AoVLGIxJLhGW2Zfu2JnDmsC3e9u5Yr/5FF3iEtdNUUnpyTzcbdhdx+7hCSE5r3gHqFkI6JmFm8mS0FcoAZ7j6/yv5RQA93nx7KOERiTWpKIg9ePIo7Jg9l5tpczn3wY1Zt19VbobQzr4gH3l/PpGM6c8qg9HCHEzFCmkTcvczdRwIZwPFmNqxin5nFAfcCN9Z2HDO70syyzCwrNzc3dAGLRJGK7q0X/mscxaVlnP/IHF5ZtDXcYcWsP7y1mtJy57ZzhoQ7lIjSJFdnuft+YCZwRqXNqcAwYKaZbQLGAW9UN7ju7o+5e6a7Z6alNb/5+kVqMqZXB6Zf+w1G9WzHjS8tY9qbmsixsc3fuIc3lm3npyf3o2fH5rXoVG1CeXVWmpm1Cz5vAUwC1lTsd/c8d+/k7r3dvTcwD5js7lmhikkkVqWlJvPMFWP50YTePDknm8uf1jhJYykvd25/YyXd27XgqpP7hTuciBPKlkhX4EMzWw4sJDAmMt3MppnZ5BCeV6RZSoiP4/Zzh/Kn7wxn7me7Of/hOWzMLaj9jVKjWetzWbMzn5tOH0iLJA2mV2XRdvdrZmamZ2WpsSJSk/kb93DVs4spKSvnljMHM+W4nsTH6Z6Ghrj8bwtZvjWPT245laSE6L0/28wWuXuj34sXvf8jInJEY/t25PWfTeCYrm249V+fcu6DH7MgWzcn1tfmPYV8uDaHS8b2jOoEEkr6XxGJUT06tOSFK8fx0CWj2H/wMBc9OpdrnltMTr5uyaqrf8zdTLwZl45tnuun14WSiEgMMzPOObYb7994CtdNGsCMVbs46/6PmbNhd7hDi3gHD5fyYtYWzhjWpVkvOlUbJRGRZqBFUjzXTRrIG9ecSLuWiUz963zueW8tZZoy5YheW7KdA0Wlmt6kFkoiIs3IoC6pvHHNBC4YncEDH2zgksfnsXJ7nqaXr8LdefqTTQzp2obMXpqptyZKIiLNTMukBP7nwhHcfeEIVmzL4+wHPua0e2fx4Pvr2bynMNzhRYR5G/eydlc+l53QSzP11iIh3AGISHhcMCaDiYPTeWvFDt5Yup27Z6zj7hnr6NY2hb5premb1oq+nVpx0sA0+qa1Dne4TervczfRrmUi541s3uun14WSiEgz1qFVElPH9WLquF5s23+It1fsYOX2A2zMLeDVxdsoKC4lPs74/rheXDdpAO1aJoU75JDbtLuQ91bt4ooT+zT7pW/rQklERADo3q4FP/5G3y9euzvb84p45MMN/H3uJv61ZBvXTxrApeN6kRgfuz3hf3hrNckJcfz4xD7hDiUqxO5PgogcFTOje7sW/P784fz7599gaLc2/PbNVZz/yBx25B0Kd3gh8clnu3lv1S6uPqUf6bqst06URESkVsd0bcOzPx7LI5eOZtPug5z30ByWb90f7rAaVWU1izgAAAz/SURBVFm5c+f01V9rkUnNlEREpE7MjLOGd+Xlq8aTGB/HRY/O5e0VO8IdVqN5edEWVu84wK/OHKyxkHpQEhGRehncpQ2vXzOBIV3bcNWzi3n4ww1Rf59JQXEpd727jtE923HusV3DHU5UURIRkXrr1DqZ534yjvNGduOud9fyh7dWR3UieeTDDewuKOa2c4fqvpB60tVZItIgKYnx3HvRSNq3TOLx2dkUFJfyu28Pj7op51duz+OJj7M5f1R3RvZoF+5woo6SiIg0WFyccfu5Q2idnMBDH24gv6iUe783MmouAV6xNY+pf51Pp1ZJ3HLm4HCHE5WURETkqJgZN50+iNYpCfzp7TUcKCrltnOG0D89su9yX/L5Pn7w5ALatkjknz8Zp5l6Gyg6vi6ISMT76cn9+MP5w1mQvYfT7v2Iq59dxKfb8sIdVrWyNu3l+39dQIdWSbzwX+Pp0aFluEOKWmqJiEijuWRsT04f2pmn5mzi6bmbeGvFTr4xoBNnDe/KKYPS6Nq2RbhD5K0VO7jppWV0aZPCcz8ZR5e2aoEcDa2xLiIhcaCohGfmbebZeZ+zbX/gDvdjurbhm4PTueLEPrRv1bTzcOUdLOH2Nz7ltaXbOTajLU/8ILNZ3ZUeqjXWQ5ZEzCwFmAUkE2jxvOzut1cpcwPwY6AUyAUud/fNNR1XSUQkurg763MK+GBNDh+uySFr8z46tEriLxccy8TB6U0Sw6x1udz88nJyC4q59tT+/Gxi/6gZ/G8s0ZhEDGjl7gVmlgh8DPzC3edVKjMRmO/uB83sKuAUd/9eTcdVEhGJbiu353HDC8tYuyufKcf14DfnBK7uCoXDpeX86e01PDknm/7prbnnohEcm9E8L+MNVRIJWSr2gILgy8Tgw6uU+dDdDwZfzgMyQhWPiESGod3a8sa1E/jpyf0Ca5jfN4tFm/c1+nm27D3IhY/O5ck52fzwhN5Mv/bEZptAQimk7TkzizezpUAOMMPd59dQ/Arg7VDGIyKRITkhnlvOHMxLPx1PnBnfe3QuT36c3Wh3vb+3cidnPzCbjbkF/N/U0fx28lDNhxUiTTKwbmbtgH8B17r7p9XsnwpcA5zs7sXV7L8SuBKgZ8+eYzZvrnHYRESiSN6hEm56aRkzVu3irOFd+PMFx5Kaksim3YW8s3InM1btYmdeEYfLyikpK+dwaTlDurbh+tMGMqF/p68cK3t3IQ9+sJ5XF29jePe2PHzJaHp21OW7EIVjIl87kdntQKG7/0+V7ZOABwkkkJzajqMxEZHY4+48Nmsjf3l3LT3atyAlMZ41O/MBGN69LQM7p5KUYCTFxxEXZ7z76U625xUxvm9HbvzWQDq0SuKhDzbw2tJtJCXEcdkJvbnhtIEkJ6j1USHqkoiZpQEl7r7fzFoA7wF/dvfplcqMAl4GznD39XU5rpKISOyav3EP//2vFbRrmcSZw7pw+tAu1d4IWFRSxj8XfM7DH37G7oJizCA5IY6pY3tx5cl9SU9tPpfu1lU0JpFjgaeBeAJjLy+6+zQzmwZkufsbZvYfYDhQsSjB5+4+uabjKomISIWDh0t5bv7nHCgq5fvjepGWmhzukCJW1CWRUFESERGpv6i7xFdERGKfkoiIiDSYkoiIiDSYkoiIiDSYkoiIiDSYkoiIiDSYkoiIiDSYkoiIiDRY1N1saGa5QNUZGNsCVRdzrrqtptcVzytv6wTsbmCY1cVTnzL1rU9tz0Ndl5rKRVJdaoqzLmX0c6bPpq4i8bNp5e5ptUZeX+4e9Q/gsdq21fS64nmVbVmNGU99ytS3PrU9D3VdaioXSXU52s9GP2f6bJrrZ1PTI1a6s96sw7aaXr95hDKNGU99ytS3PnV53lB1PcaRykVSXep6nMaqS9XX+jmrmT6bmveF87M5oqjrzmoqZpblIZhnJhxUl8gVS/WJpbpAbNUnlHWJlZZIKDwW7gAakeoSuWKpPrFUF4it+oSsLmqJiIhIg6klIiIiDRbzScTMnjSzHDP72trudXjvGDNbYWYbzOwBM7NK+641s7VmttLM/tK4UdcYU6PXx8x+a2bbzGxp8HFW40debTwh+WyC+28yMzezTkc6RmML0Wdzp5ktD34u75lZt8aPvNp4QlGXu8xsTbA+/zKzdo0febXxhKIuFwZ/98vNLOTjJkdThyMc7zIzWx98XFZpe42/V9UK1WVfkfIATgJGA5824L0LgPGAAW8DZwa3TwT+AyQHX6dHeX1+C9wUC59NcF8P4F0C9xN1iub6AG0qlfk58H9RXJdvAQnB538msFx2tNblGGAQMBPIjNQ6BOPrXWVbB2Bj8N/2wefta6pvTY+Yb4m4+yxgb+VtZtbPzN4xs0VmNtvMBld9n5l1JfALPNcD/7t/B74d3H0V8Cd3Lw6eIye0tfhSiOoTFiGsy73AzUCTDviFoj7ufqBS0VY0UZ1CVJf33L00WHQekBHaWgSEqC6r3X1tU8QfPF+D6nAEpwMz3H2vu+8DZgBnNPRvRMwnkSN4DLjW3ccANwGPVFOmO7C10uutwW0AA4FvmNl8M/vIzI4LabS1O9r6AFwT7GZ40szahy7UWh1VXcxsMrDN3ZeFOtA6OurPxsx+b2ZbgEuB20IYa20a4+eswuUEvumGS2PWJVzqUofqdAe2VHpdUa8G1TehjieNGWbWGjgBeKlSd19ydUWr2VbxLTCBQDNwHHAc8KKZ9Q1m7ybVSPX5X+DO4Os7gbsJ/JI3qaOti5m1BG4l0G0Sdo302eDutwK3mtmvgWuA2xs51Fo1Vl2Cx7oVKAWebcwY66ox6xIuNdXBzH4E/CK4rT/wlpkdBrLd/XyOXK8G1bfZJRECra/97j6y8kYziwcWBV++QeAPa+XmdgawPfh8K/BqMGksMLNyAnPT5IYy8CM46vq4+65K73scmB7KgGtwtHXpB/QBlgV/sTKAxWZ2vLvvDHHs1WmMn7XKngP+TRiSCI1Ul+Ag7jnAN8PxpSuosT+XcKi2DgDu/hTwFICZzQR+6O6bKhXZCpxS6XUGgbGTrTSkvqEeEIqEB9CbSgNSwCfAhcHnBow4wvsWEmhtVAwynRXc/lNgWvD5QAJNQ4vi+nStVOZ64PlorUuVMptowoH1EH02AyqVuRZ4OYrrcgawCkhrys8klD9nNNHAekPrwJEH1rMJ9Ka0Dz7vUJf6VhtXU3+YYfjh+SewAyghkGmvIPBt9R1gWfCH+rYjvDcT+BT4DHiIL2/OTAKeCe5bDJwa5fX5B7ACWE7gG1jXaK1LlTKbaNqrs0Lx2bwS3L6cwDxI3aO4LhsIfOFaGnw01ZVmoajL+cFjFQO7gHcjsQ5Uk0SC2y8Pfh4bgB/VVt+aHrpjXUREGqy5Xp0lIiKNQElEREQaTElEREQaTElEREQaTElEREQaTElEop6ZFTTx+Z4wsyGNdKwyC8zQ+6mZvVnbzLZm1s7Mrm6Mc4s0Bl3iK1HPzArcvXUjHi/Bv5woMKQqx25mTwPr3P33NZTvDUx392FNEZ9IbdQSkZhkZmlm9oqZLQw+JgS3H29mn5jZkuC/g4Lbf2hmL5nZm8B7ZnaKmc00s5ctsAbGsxVrKwS3ZwafFwQnSFxmZvPMrHNwe7/g64VmNq2OraW5fDmRZGsze9/MFltgfYfzgmX+BPQLtl7uCpb9ZfA8y83sjkb8bxSplZKIxKr7gXvd/TjgAuCJ4PY1wEnuPorAjLh/qPSe8cBl7n5q8PUo4DpgCNAXmFDNeVoB89x9BDAL+Eml898fPH+t8w8F5236JoEZAwCKgPPdfTSB9WvuDiaxW4DP3H2ku//SzL4FDACOB0YCY8zspNrOJ9JYmuMEjNI8TAKGVJrhtI2ZpQJtgafNbACBGUoTK71nhrtXXrNhgbtvBTCzpQTmLvq4ynkO8+WElYuA04LPx/PlWgzPAf9zhDhbVDr2IgJrO0Bg7qI/BBNCOYEWSudq3v+t4GNJ8HVrAkll1hHOJ9KolEQkVsUB4939UOWNZvYg8KG7nx8cX5hZaXdhlWMUV3peRvW/LyX+5cDikcrU5JC7jzSztgSS0c+ABwisHZIGjHH3EjPbBKRU834D/ujuj9bzvCKNQt1ZEqveI7D2BgBmVjFldltgW/D5D0N4/nkEutEAptRW2N3zCCx/e5OZJRKIMyeYQCYCvYJF84HUSm99F7g8uL4EZtbdzNIbqQ4itVISkVjQ0sy2VnrcQOAPcmZwsHkVgen7Af4C/NHM5gDxIYzpOuAGM1sAdAXyanuDuy8hMCPrFAILNmWaWRaBVsmaYJk9wJzgJcF3uft7BLrL5prZCuBlvppkREJKl/iKhEBwlcVD7u5mNgW42N3Pq+19ItFGYyIioTEGeCh4RdV+wrDcsEhTUEtEREQaTGMiIiLSYEoiIiLSYEoiIiLSYEoiIiLSYEoiIiLSYEoiIiLSYP8f1Q3qvvf/DBEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()\n",
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='9' class='' max='50', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      18.00% [9/50 03:49<17:26]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.013176</td>\n",
       "      <td>0.618647</td>\n",
       "      <td>0.873200</td>\n",
       "      <td>00:24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.019301</td>\n",
       "      <td>0.608411</td>\n",
       "      <td>0.874500</td>\n",
       "      <td>00:24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.014788</td>\n",
       "      <td>0.622957</td>\n",
       "      <td>0.870900</td>\n",
       "      <td>00:25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.017873</td>\n",
       "      <td>0.640418</td>\n",
       "      <td>0.869100</td>\n",
       "      <td>00:24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.024040</td>\n",
       "      <td>0.638744</td>\n",
       "      <td>0.869400</td>\n",
       "      <td>00:26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.030291</td>\n",
       "      <td>0.641987</td>\n",
       "      <td>0.868900</td>\n",
       "      <td>00:25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.039610</td>\n",
       "      <td>0.659848</td>\n",
       "      <td>0.862400</td>\n",
       "      <td>00:24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.050679</td>\n",
       "      <td>0.603996</td>\n",
       "      <td>0.863400</td>\n",
       "      <td>00:26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.053857</td>\n",
       "      <td>0.633895</td>\n",
       "      <td>0.859200</td>\n",
       "      <td>00:24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='progress-bar-interrupted' max='781', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      Interrupted\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with accuracy value: 0.873199999332428.\n",
      "Better model found at epoch 1 with accuracy value: 0.8744999766349792.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-caa9e0bbe1f3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_one_cycle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_lr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1e-4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSaveModelCallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmonitor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'accuracy'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'max'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/pyt/lib/python3.7/site-packages/fastai/train.py\u001b[0m in \u001b[0;36mfit_one_cycle\u001b[0;34m(learn, cyc_len, max_lr, moms, div_factor, pct_start, final_div, wd, callbacks, tot_epochs, start_epoch)\u001b[0m\n\u001b[1;32m     21\u001b[0m     callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor, pct_start=pct_start,\n\u001b[1;32m     22\u001b[0m                                        final_div=final_div, tot_epochs=tot_epochs, start_epoch=start_epoch))\n\u001b[0;32m---> 23\u001b[0;31m     \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcyc_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_lr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m def fit_fc(learn:Learner, tot_epochs:int=1, lr:float=defaults.lr,  moms:Tuple[float,float]=(0.95,0.85), start_pct:float=0.72,\n",
      "\u001b[0;32m~/anaconda3/envs/pyt/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[1;32m    198\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallback_fns\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callback_fns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m         \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    201\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pyt/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(epochs, learn, callbacks, metrics)\u001b[0m\n\u001b[1;32m     99\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0myb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpbar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m                 \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pyt/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mloss_batch\u001b[0;34m(model, xb, yb, loss_func, opt, cb_handler)\u001b[0m\n\u001b[1;32m     32\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mopt\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mskip_bwd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_backward_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mskip_bwd\u001b[0m\u001b[0;34m:\u001b[0m                     \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_backward_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_step_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m     \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pyt/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    148\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m         \"\"\"\n\u001b[0;32m--> 150\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pyt/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     97\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     98\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(50, max_lr = 1e-4, callbacks=[callbacks.SaveModelCallback(learn, monitor = 'accuracy', mode = 'max')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('resnet34_cifar10_bs64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "87.51\n"
     ]
    }
   ],
   "source": [
    "learn = cnn_learner(data, models.resnet34, metrics = accuracy, pretrained = True)\n",
    "learn.load('resnet34_cifar10_bs64')\n",
    "preds, y, losses = learn.get_preds(with_loss=True, ds_type = DatasetType.Valid)\n",
    "interp = ClassificationInterpretation(learn, preds, y, losses)\n",
    "conf_mat = interp.confusion_matrix()\n",
    "acc = np.trace(conf_mat) / np.sum(conf_mat, axis = None)\n",
    "print(acc * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (pyt)",
   "language": "python",
   "name": "pyt"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
